/***************************************************************************
 *   Copyright (C) 2004 by Tim Macfarlane                                  *
 *   timmacfarlane@blueyonder.co.uk                                        *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/

#include "StdAfx.h"

#include "parser.h"
#include <assert.h>

#include <math.h>
#include <iostream>

using namespace std;

#define NEWLINE_OPERATOR ";"
#define TOKEN_BUFFER_LENGTH 1024
#define INDENT_SYNTAX 1
#define NO_VERSION (static_cast<unsigned short> (-1))
#define INDENT_ERROR (static_cast<uint> (-1))

#if defined (WIN32) | defined (MACOS)
char *strndup (const char *str, size_t n) {
	char *result = new char [n + 1];
	strncpy_s (result, n + 1, str, n);
	return result;
}
#endif

#ifdef RUBY
VALUE tt_brace, tt_parenthesis, tt_list, tt_bracket;
VALUE sNew;
VALUE cSourceLocation;
VALUE cTokens;
VALUE cIdentifierToken;
VALUE cIntegerToken;
VALUE cFloatingToken;
VALUE cStringToken;
VALUE cQuoteToken;
VALUE cErrorToken;
VALUE cTokenList;
VALUE cVersion;
VALUE cVersionToken;

VALUE sl_type_file, sl_type_source;

ID iPush;

VALUE cBlock;
VALUE class_new (VALUE cls) {
	return rb_funcall (cls, sNew, 0);
}
#endif

#ifdef JAVA
jclass SourceDescriptionClass,
	   SourceLocationClass,
	   IntegerTokenClass,
	   IdentifierTokenClass,
	   BracketTokenClass,
	   ArrayListClass,
	   FloatingTokenClass,
	   StringTokenClass,
	   QuoteTokenClass,
	   ErrorTokenClass,
	   VersionTokenClass,
	   VersionClass;

jmethodID SourceDescriptionConstructor,
		  SourceDescriptionGetFilename,
		  SourceDescriptionGetIsFilename,
		  SourceLocationConstructor,
		  IntegerTokenConstructor,
		  IdentifierTokenConstructor,
		  BracketTokenConstructor,
		  ArrayListConstructor,
		  ArrayListAdd,
		  FloatingTokenConstructor,
		  StringTokenConstructor,
		  QuoteTokenConstructor,
		  ErrorTokenConstructor,
		  VersionTokenConstructor1,
		  VersionTokenConstructor2,
		  VersionConstructor;
#endif

char *token_buffer;

bool is_whitespace (int c);

Token::Token () {
	_sloc = NULL;
	_attributes = NULL;
}

Token::~Token () {
	if (_attributes) {
		for (vector<Token*>::iterator t = _attributes->begin (); t != _attributes->end (); t++) {
			delete *t;
		}

		delete _attributes;
	}

	if (_sloc)
		delete _sloc;
}

#ifdef RUBY
VALUE Token::attributes_to_ruby () {
	if (_attributes) {
		VALUE attr_array = rb_ary_new2 (_attributes->size ());
		for (uint n = 0; n < _attributes->size (); n++)
			rb_ary_store (attr_array, n, (*_attributes)[n]->to_ruby ());

		return attr_array;
	} else {
		return Qnil;
	}
}

VALUE Token::to_ruby () {
	VALUE tok = token_to_ruby ();

	rb_iv_set (tok, "@attributes", attributes_to_ruby ());
	if (_sloc)
		rb_iv_set (tok, "@sloc", _sloc->to_ruby ());
	else
		rb_iv_set (tok, "@sloc", Qnil);
	return tok;
}
#endif

#ifdef JAVA
jobject Token::attributes_to_java (JavaEnvironment *env) {
	if (_attributes) {
		jobject list = env->createArrayList ();
		for (uint n = 0; n < _attributes->size (); n++) {
			env->ArrayList_Add (list, (*_attributes)[n]->to_java (env));
		}

		return list;
	} else {
		return NULL;
	}
}

jobject Token::to_java (JavaEnvironment *env) {
	jobject sloc = _sloc->to_java (env);
	jobject attrs = attributes_to_java (env);

	return token_to_java (env, attrs, sloc);
}
#endif

#ifdef CLR

System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^Token::attributes_to_clr () {
	System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^list = gcnew System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ();

	if (_attributes) {
		for (uint n = 0; n < _attributes->size (); n++) {
			list->Add ((*_attributes) [n]->to_clr ());
		}

		return list;
	} else {
		return nullptr;
	}
}

Tycho::Parser::Tokens::Token ^Token::to_clr () {
	Tycho::Parser::Tokens::SourceLocation ^sloc = _sloc->to_clr ();
	System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs = attributes_to_clr ();

	return token_to_clr (attrs, sloc);
}

#endif

IntegerToken::IntegerToken (int i) {
	_value = i;
}

#ifdef RUBY
VALUE IntegerToken::token_to_ruby () {
	VALUE it = class_new (cIntegerToken);
	rb_iv_set (it, "@value", INT2NUM (_value));
	return it;
}
#endif

#ifdef JAVA
jobject IntegerToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	return env->createIntegerToken (_value, attrs, sloc);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^IntegerToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::IntegerToken (attrs, sloc, _value);
}

#endif

IdentifierToken::IdentifierToken (const char *i, bool is_op) {
	_id = i;
	_module_path = NULL;
	_is_operator = is_op;
}

IdentifierToken::~IdentifierToken () {
	if (_module_path) {
		for (vector<const char*>::iterator m = _module_path->begin (); m != _module_path->end (); m++) {
			free (const_cast<char*> (*m));
		}
		delete _module_path;
	}
	free (const_cast<char*> (_id));
}

#ifdef RUBY
VALUE IdentifierToken::token_to_ruby () {
	VALUE id = class_new (cIdentifierToken);
	rb_iv_set (id, "@value", rb_str_intern (rb_str_new2 (_id)));
	if (_module_path)
		rb_iv_set (id, "@module", module_to_ruby ());
	return id;
}

VALUE IdentifierToken::module_to_ruby () {
	VALUE module_array = rb_ary_new2 (_module_path->size ());
	for (uint n = 0; n < _module_path->size (); n++)
		rb_ary_store (module_array, n, rb_str_intern (rb_str_new2 ((*_module_path)[n])));
	return module_array;
}
#endif

#ifdef JAVA
jobject IdentifierToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	return env->createIdentifierToken (env->createString (_id), attrs, sloc);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^IdentifierToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::IdentifierToken (attrs, sloc, gcnew System::String (_id), module_to_clr (), _is_operator);
}

System::Collections::Generic::List<System::String^> ^IdentifierToken::module_to_clr () {
	if (_module_path) {
		System::Collections::Generic::List<System::String^> ^list = gcnew System::Collections::Generic::List<System::String^> ();

		for (vector<const char*>::const_iterator i = _module_path->begin (); i != _module_path->end (); i++) {
			list->Add (gcnew System::String (*i));
		}

		return list;
	} else {
		return nullptr;
	}
}

#endif

FloatToken::FloatToken (double v) {
	_value = v;
}

#ifdef RUBY
VALUE FloatToken::token_to_ruby () {
	VALUE t = class_new (cFloatingToken);
	rb_iv_set (t, "@value", rb_float_new (_value));
	return t;
}
#endif

#ifdef JAVA
jobject FloatToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	return env->createFloatingToken (_value, attrs, sloc);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^FloatToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::FloatToken (attrs, sloc, _value);
}

#endif

ListToken::ListToken (Type type, vector<Token*> *tokens) {
	_type = type;
	_tokens = tokens;
}

ListToken::~ListToken () {
	if (_tokens) {
		for (vector<Token*>::iterator t = _tokens->begin (); t != _tokens->end (); t++) {
			delete *t;
		}

		delete _tokens;
	}
}

#ifdef RUBY
VALUE ListToken::type_to_ruby () {
	switch (_type) {
		case Brace:
			return tt_brace;
		case Parenthesis:
			return tt_parenthesis;
		case Bracket:
			return tt_bracket;
		case List:
			return tt_list;
		default:
			assert (false); // didn't expect this list token type
			return tt_list;
	}
}

VALUE ListToken::token_to_ruby () {
	VALUE token_ary = class_new (cTokens);
	for (uint n = 0; n < _tokens->size (); n++)
		rb_funcall (token_ary, iPush, 1, (*_tokens)[n]->to_ruby ());

	return rb_funcall (cTokenList, rb_intern ("new"), 2, type_to_ruby (), token_ary);
}
#endif

#ifdef JAVA
jobject ListToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	jobject list = env->createArrayList ();

	for (uint n = 0; n < _tokens->size (); n++) {
		env->ArrayList_Add (list, (*_tokens)[n]->to_java (env));
	}

	if (_type == List) {
		return list;
	} else {
		jint javaType;

		switch (_type) {
			case Brace:
				javaType = 1;
				break;
			case Parenthesis:
				javaType = 2;
				break;
			case Bracket:
				javaType = 3;
				break;
			default:
				assert (false); // didn't expect this list token type
				javaType = 1;
		}

		jobject brackets = env->createBracketToken (javaType, list, attrs, sloc);
		return brackets;
	}
}
#endif

#ifdef CLR

System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^ListToken::to_clr_tokens () {
	System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^list = gcnew System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ();
	
	for (uint n = 0; n < _tokens->size (); n++) {
		list->Add ((*_tokens) [n]->to_clr ());
	}

	return list;
}

Tycho::Parser::Tokens::Token ^ListToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^list = to_clr_tokens ();

	Tycho::Parser::Tokens::BracketType type;

	switch (_type) {
		case Brace:
			type = Tycho::Parser::Tokens::BracketType::Brace;
			break;
		case Parenthesis:
			type = Tycho::Parser::Tokens::BracketType::Parenthesis;
			break;
		case Bracket:
			type = Tycho::Parser::Tokens::BracketType::Bracket;
			break;
		default:
			assert (false); // didn't expect this list token type
			type = Tycho::Parser::Tokens::BracketType::Brace;
	}

	return gcnew Tycho::Parser::Tokens::BracketToken (attrs, sloc, list, type);
}

#endif

StringToken::StringToken (const char *label, const char *str, bool escaped) {
	_label = label;
	_str = str;
	_escaped = escaped;
}

StringToken::~StringToken () {
	free (const_cast<char*> (_label));
	free (const_cast<char*> (_str));
}

#ifdef RUBY
VALUE StringToken::token_to_ruby () {
	VALUE t = class_new (cStringToken);
	rb_iv_set (t, "@string_type", rb_str_intern (rb_str_new2 (_label)));
	rb_iv_set (t, "@value", rb_str_new2 (_str));
	return t;
};
#endif

#ifdef JAVA
jobject StringToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	return env->createStringToken (env->createString (_str), attrs, sloc);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^StringToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::StringToken (attrs, sloc, gcnew System::String (_str), gcnew System::String (_label), _escaped);
}

#endif

ErrorToken::ErrorToken (const char *code) {
	_code = code;
}

#ifdef RUBY
VALUE ErrorToken::token_to_ruby () {
	VALUE t = class_new (cErrorToken);
	rb_iv_set (t, "@value", rb_str_intern (rb_str_new2 (_code)));
	return t;
};
#endif

#ifdef JAVA
jobject ErrorToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	return env->createErrorToken (env->createString (_code), attrs, sloc);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^ErrorToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::ErrorToken (attrs, sloc, gcnew System::String (_code));
}

#endif

Version::Version () {
	major = NO_VERSION;
	minor = NO_VERSION;
	revision = NO_VERSION;
	build = NO_VERSION;
}

#ifdef RUBY
VALUE Version::to_ruby () {
	VALUE t = class_new (cVersion);

	if (major != NO_VERSION)
		rb_iv_set (t, "@major", INT2FIX (major));
	if (minor != NO_VERSION)
		rb_iv_set (t, "@minor", INT2FIX (minor));
	if (revision != NO_VERSION)
		rb_iv_set (t, "@revision", INT2FIX (revision));
	if (build != NO_VERSION)
		rb_iv_set (t, "@build", INT2FIX (build));

	return t;
}
#endif

#ifdef JAVA
jobject Version::to_java (JavaEnvironment *env) {
	return env->createVersion (major, minor, revision, build);
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Version ^Version::to_clr () {
	return gcnew Tycho::Parser::Tokens::Version (major, minor, revision, build);
}

#endif

VersionToken::VersionToken () {
	_has_to_version = false;
	_branch = NULL;
}

#ifdef RUBY
VALUE VersionToken::token_to_ruby () {
	VALUE t = class_new (cVersionToken);

	rb_iv_set (t, "@version", _version.to_ruby ());

	if (_has_to_version) {
		rb_iv_set (t, "@to_version", _to_version.to_ruby ());
	}

	if (_branch) {
		rb_iv_set (t, "@branch", rb_str_new2 (_branch));
	}

	return t;
}
#endif

#ifdef JAVA
jobject VersionToken::token_to_java (JavaEnvironment *env, jobject attrs, jobject sloc) {
	if (_has_to_version || _branch) {
		return env->createVersionToken (_version.to_java (env),
										_has_to_version? _to_version.to_java (env): NULL, 
										_branch? env->createString (_branch): NULL,
										attrs,
										sloc);
	} else {
		return env->createVersionToken (_version.to_java (env),
										attrs,
										sloc);
	}
}
#endif

#ifdef CLR

Tycho::Parser::Tokens::Token ^VersionToken::token_to_clr (System::Collections::Generic::List<Tycho::Parser::Tokens::Token^> ^attrs, Tycho::Parser::Tokens::SourceLocation ^sloc) {
	return gcnew Tycho::Parser::Tokens::VersionToken (
		attrs,
		sloc,
		_version.to_clr (),
		_has_to_version? _to_version.to_clr (): nullptr,
		_branch? gcnew System::String (_branch): nullptr);
}

#endif

char *indent_line_get_line (Line *line);

SourceLocation *TokenLocation::calc_location (char *line) {
	return new SourceLocation (_lineno, _lineno, _token_start - _line_start + _colno, line - _line_start + _colno, _source, _file_name);
//	return new SourceLocation (_lineno, _lineno, _token_start - _line_start, line - _line_start, _source, _file_name);
}

SourceLocation *TokenLocation::calc_location (char *line, uint end_lineno, char *line_start) {
	return new SourceLocation (_lineno, end_lineno, _token_start - _line_start + _colno, line - line_start + _colno, _source, _file_name);
//	return new SourceLocation (_lineno, end_lineno, _token_start - _line_start, line - line_start, _source, _file_name);
}

SourceLocation *TokenLocation::new_location (char *source, char *file_name, uint line, uint col) {
	return new SourceLocation (line, line, col, col, source, file_name);
}

SourceLocation::SourceLocation (uint line_start, uint line_end, uint col_start, uint col_end, char *source, char *file_name) {
	_line_start = line_start;
	_line_end = line_end;
	_col_start = col_start;
	_col_end = col_end;
	_source = source;
	_file_name = file_name;
}

#ifdef RUBY
VALUE SourceLocation::to_ruby () {
	char *type;
	switch (_type) {
		case Source:
			type = "source";
			break;
		case File:
			type = "file";
			break;
		case None:
			type = "none";
			break;
		default:
			assert (false); // don't know this type
	}

	VALUE source;
	if (_source)
		source = rb_str_new2 (_source);
	else
		source = Qnil;

	return rb_funcall (cSourceLocation, sNew,
					   5,
					   INT2NUM (_line_start),
					   INT2NUM (_line_end),
					   INT2NUM (_col_start),
					   INT2NUM (_col_end),
					   rb_ary_new3 (2, rb_str_intern (rb_str_new2 (type)), source));
}
#endif

#ifdef JAVA
jobject SourceLocation::to_java (JavaEnvironment *env) {
	jobject desc = env->getSourceDescription ();
	return env->createSourceLocation (desc, _line_start, _line_end, _col_start, _col_end);

	/*
	//jobject desc = env->NewObject (SourceDescriptionClass, SourceDescriptionConstructor, (char) (_type == File), _source? env->NewStringUTF (_source): NULL);
	//return env->NewObject (SourceLocationClass, SourceLocationConstructor, desc, (int) _line_start, (int) _line_end, (int) _col_start, (int) _col_end);

	jstring source = NULL;

	if (_source) {
		cerr << "source is: " << _source << endl;
		source = env->NewStringUTF (_source);
	}
	jobject desc = env->NewObject (SourceDescriptionClass, SourceDescriptionConstructor, (jboolean) 0, (jobject) NULL);
	return env->NewObject (SourceLocationClass, SourceLocationConstructor, (jobject) NULL, (int) _line_start, (int) _line_end, (int) _col_start, (int) _col_end);
	return NULL;
	*/
}
#endif

#ifdef CLR
Tycho::Parser::Tokens::SourceLocation ^SourceLocation::to_clr () {
	Tycho::Parser::Tokens::SourceLocation ^si = gcnew Tycho::Parser::Tokens::SourceLocation (gcnew System::String (_source), _file_name? gcnew System::String (_file_name): nullptr, _line_start, _line_end, _col_start, _col_end);
	return si;
}
#endif

void init_tokenizer () {
	token_buffer = new char [TOKEN_BUFFER_LENGTH + 1];
}

Block::~Block () {
	clear ();
}

void block_free (Block *b) {
	delete b;
}

#ifdef RUBY
VALUE Block::to_ruby () {
	VALUE block_value;
	block_value = Data_Wrap_Struct (cBlock, 0, block_free, this);
	rb_iv_set (block_value, "@type", rb_str_intern (rb_str_new2 ("block")));
	rb_iv_set (block_value, "@lexed", Qnil);
	rb_iv_set (block_value, "@source", Qnil);
	return block_value;
}
#endif

char *Block::get_line (int n) {
	Line *line = _first;

	while (line && n) {
		line = line->_next;
		n--;
	}

	if (line)
		return line->_whole_line;
	else
		return NULL;
}

uint Block::get_indent (const char *line, uint *n_indent_chars, bool *error) {
	uint len = 0, match_len = 0;
	while (line[len] == '\t' || line[len] == ' ') {
		if ((len < _indent_len) && (line[len] == _indent[match_len]) && (len == match_len)) {
			match_len++;
		}

		len++;
	}

	*n_indent_chars = len;
	*error = false;

	if (match_len < len && match_len < _indent_len) {
		*error = true;
		return _indent_len;
	} else if (_indent_len == len && match_len == len) {
		return len;
	} else if (match_len == len) {
		return len;
	} else if (match_len == _indent_len) {
		_indent_len = len;
		_indent = line;
		return len;
	} else {
		assert (false);		// didn't expect this.
		*error = true;
		return _indent_len;
	}
}

/*
uint get_indent_old (const char *line, uint *n_indent_chars) {
	uint len = 0;
	const char *line_i = line;
	while (*line_i == '\t' || *line_i == ' ') {
		line_i++;
		len++;
	}

	if (len == 0 && (*line_i == '\n' || *line_i == 0)) {
		// if indent len is 0, the indent is obviously 0 too.
		// but we don't clear the stack because this line
		// is empty, which doesn't effect the indent level
		return 0;
	}

	if (_indents.size () == 0) {
		if (len > 0) {
			Indent indent;
			indent.indent = line;
			indent.len = len;
			_indents.push_back (indent);
			return _indents.size ();
		} else {
			return 0;
		}
	} else {
		uint top_len = _indents.back ().len;
		if (top_len < len) {
			const char *top_indent = _indents.back ().indent;

			for (uint n = 0; n < top_len; n++) {
				if (top_indent[n] != line[n])
					return INDENT_ERROR;
			}

			Indent indent;
			indent.indent = line;
			indent.len = len;
			_indents.push_back (indent);
			return _indents.size ();
		} else if (top_len == len) {
			const char *top_indent = _indents.back ().indent;

			for (uint n = 0; n < top_len; n++) {
				if (top_indent[n] != line[n])
					return INDENT_ERROR;
			}

			return _indents.size ();
		} else if (len == 0) {
			// indent len is 0, clear the stack.
			
			_indents.clear ();

			return 0;
		} else {
			uint cur_indent_level = _indents.size () - 1;

			for (vector<Indent>::reverse_iterator i = _indents.rbegin () + 1; i != _indents.rend (); i++) {
				const char *i_indent = i->indent;
				uint i_len = i->len;

				if (i_len == len) {
					for (uint n = 0; n < i_len; n++) {
						if (i_indent[n] != line[n])
							return INDENT_ERROR;
					}

					_indents.erase (_indents.begin () + cur_indent_level, _indents.end ());
					return cur_indent_level;
				}

				cur_indent_level--;
			}

			return INDENT_ERROR;
		}
	}
}
*/

/**
 * Allocates a new string, initialising it with the content between the start
 * and end pointers (not including the character pointed to by end).
 * \param start The start of the content
 * \param end The end of the content
 * \return A new string containing [start, end).
 */
char *make_line (const char *start, const char *end) {
	return strndup (start, end - start);
}

void free_line (Line *line) {
	while (line) {
		Line *this_line = line;
		line = line->_next;

		delete this_line;
	}
}

Line::~Line () {
	free (_whole_line);
}

Block::Block () {
	construct ();
}

void Block::construct () {
	_first = NULL;
	_last = NULL;
	_size = 0;
	_lineno = 1;
	_source = NULL;
	_file_name = NULL;
	_colno = 0;
	_indent = "";
	_indent_len = 0;
}

void Block::clear () {
	if (_first) free_line (_first);
	construct ();
}

void Line::indent (int indentation) {
	Line *line = this;

	while (line) {
		int indent = line->_indent + indentation;
		if (indent < 0)
			indent = 0;
		line->_indent = indent;

		line = line->_next;
	}
}

void Line::set_indent (int indentation) {
	Line *line = this;

	while (line) {
		if (*line->_line)
			line->_indent = indentation;

		line = line->_next;
	}
}

Line::Line (int indent, char *whole_line, char *line, bool indent_error) {
	_whole_line = whole_line;
	_line = line;
	_indent = indent;
	_indent_error = indent_error;
	_next = NULL;
}

Line *Line::clone () {
	char *new_whole_line = _strdup (_whole_line);
	char *new_line = new_whole_line + (_line - _whole_line);
	return new Line (_indent, new_whole_line, new_line, _indent_error);
}

void Block::add_line (Line *indent_line) {
	//if (indent_line->_indent_error)
		//cout << '-';

	//cout << indent_line->_indent << ':' << indent_line->_line << ':' << endl;

	if (_last) {
		_last->_next = indent_line;
		_last = _last->_next;
	} else {
		_first = _last = indent_line;
	}

	_size++;
}

void Block::add_line (int indent, char *whole_line, char *line, bool indent_error) {
	add_line (new Line (indent, whole_line, line, indent_error));
}

/**
 * Parses input into lines, and each line into the line content plus it's
 * indentation. Each line is added to the lines instance variable.
 * lines is not cleared each time this method is called, so this method can be
 * used to incrementally consume source as it becomes available.
 *
 * \param input A string to consume.
 */
void Block::consume (const char *input) {
	char *c_line = NULL;
	string line;
	uint n_indent_chars = 0;
	bool indent_error;
	uint indent = get_indent (input, &n_indent_chars, &indent_error);

	const char *start = input;

	while (*input != 0) {
		if (*input == '/' && input[1] == '*') {
			line.append (start, input - start);

			input += 2;

			while (*input) {
				if (*input == '*' && input[1] == '/') {
					input += 2;
					break;
				}
				input++;
			}

			if (*input && !is_whitespace (*input) && *input != '\n') {
				// push a space to split the tokens either side of the comment
				line.append (" ");
			}

			start = input;

			if (!*input) {
				break;
			} else {
				continue;
			}
		} else if (*input == '/' && input[1] == '/') {
			line.append (start, input - start);
			c_line = make_line (start, input);
			line.clear ();
			add_line (indent, c_line, c_line + n_indent_chars, indent_error);

			input += 2;

			while (*input && *input != '\n')
				input++;

			start = input;

			if (!*input)
				break;
			else {
				input++;
				start++;
				continue;
			}
		} else if (*input == '\n') {
			const char *line_end = input;

			// check for DOS/Windows newline convention
			if (input > start && *(input - 1) == '\r') {
				line_end--;
			}

			line.append (start, line_end - start);
			c_line = _strdup (line.c_str ());
			line.clear ();
			add_line (indent, c_line, c_line + n_indent_chars, indent_error);

			const char *new_line_begin = input + 1;

			indent = get_indent (new_line_begin, &n_indent_chars, &indent_error);
			start = new_line_begin;
		} else if (*input == '\\') {
			// see if there's a newline after this character
			const char *scout = input + 1;
			while (is_whitespace (*scout)) scout++;

			if (*scout == '\n') {
				line.append (start, input - start);
				line.append (" ");
				input = scout + 1;

				// remove indent on next line
				while (is_whitespace (*input)) input++;

				start = input;
			}
		}
		input++;
	}

	line.append (start, input - start);
	c_line = _strdup (line.c_str ());
	add_line (indent, c_line, c_line + n_indent_chars, indent_error);
}

void Block::set_indent (int indentation) {
	if (_first)
		_first->set_indent (indentation);
}

void Block::indent (int indentation) {
	if (_first)
		_first->indent (indentation);
}

uint Line::get_total_length () {
	Line *lines = this;
	int length = 0;

	while (lines) {
		length += strlen (lines->_line) + lines->_indent;

		lines = lines->_next;
	}

	return length;
}

void Line::build_content (char *content, bool add_newline) {
	Line *lines = this;
	while (lines) {
		//int indent = lines->_indent;
		char *line = lines->_whole_line + lines->_indent;

		//while (indent--)
			//*(content++) = ' ';

		while (*line)
			*(content++) = *(line++);

		lines = lines->_next;

		if (lines || add_newline)
			*(content++) = '\n';
	}

	*content = '\0';
}

char *Block::get_content (bool add_newline) {
	uint content_length;
	char *content;

	int newline_room;
	if (add_newline)
		newline_room = 1;
	else
		newline_room = 0;

	content_length = _size + _first->get_total_length () + newline_room;

	content = (char*) malloc (sizeof (char) * content_length);
	_first->build_content (content, add_newline);

	return content;
}

bool is_binary (int c) {
	return c == '0' || c == '1';
}

bool is_octal (int c) {
	return c >= '0' && c <= '7';
}

bool is_hex (int c) {
	return (c >= '0' && c <= '7')
		|| (c >= 'a' && c <= 'f')
		|| (c >= 'A' && c <= 'F');
}

bool is_numeric (int c) {
	return c >= '0' && c <= '9';
}

bool is_alpha (int c) {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

bool is_first_identifier (int c) {
	return is_alpha (c)
		|| c == '_';
}

bool is_identifier (int c) {
	return is_first_identifier (c)
		|| is_numeric (c)
		|| c == '-';
}

bool is_operator (int c) {
	return c == '|'
		|| c == '&'
		|| c == '*'
		|| c == '-'
		|| c == '+'
		|| c == '/'
		|| c == '\\'
		|| c == '$'
		|| c == '!'
		|| c == '^'
		|| c == '`'
		|| c == '='
		|| c == '?'
		|| c == '#'
		|| c == '@'
		|| c == '~'
		|| c == '<'
		|| c == '>'
		|| c == ':'
		|| c == ';';
}

bool is_whitespace (int c) {
	return c == ' ' || c == '\t';
}

bool is_word (int c) {
	return is_identifier (c);
}

uint token_buffer_length (char *token_buffer) {
	uint index = 0;
	while (token_buffer [index]) {index++;}
	return index;
}

char *make_string (char *token_buffer) {
	char *s = new char [token_buffer_length (token_buffer) + 1];
	int index = 0;

	while (token_buffer [index]) {
		s [index] = token_buffer [index];
		index++;
	}
	s [index] = 0;

	return s;
}

void Block::lex (Parser *parser) {
	return lex (parser, 0, false);
}

void Block::lex (Parser *parser, char bracket_end, bool in_block) {
	if (_first) {
		parser->_sloc = new SourceLocation (1, _size, 0, strlen (_last->_line), _source, _file_name);

		char *line = _first->_line;

		Lexer lexer (this);
		lexer.lex (parser, &line, bracket_end, in_block, 0, 0, false);
	}
}

Tokens::Tokens () {
	_newline = false;
	_indent_block_last = false;
	_has_tokens = false;
	_last_identifier = NULL;
	_awaiting_attribute = false;

	_tokens = new vector<Token*>;

	_sloc = NULL;
	_newline_sloc = NULL;
	_attribute_sloc = NULL;
	_identifier_sloc = NULL;
	_module_delim_sloc = NULL;

	_block_first = false;

	_module_path = NULL;
	_attributes = NULL;
}

Tokens::~Tokens () {
	if (_newline_sloc)
		delete _newline_sloc;
	if (_attribute_sloc)
		delete _attribute_sloc;
	if (_identifier_sloc)
		delete _identifier_sloc;
	if (_module_delim_sloc)
		delete _module_delim_sloc;
	if (_last_identifier)
		delete _last_identifier;
	if (_sloc)
		delete _sloc;

	clear_tokens ();
	clear_module_path ();
	clear_attributes ();
}

void Tokens::clear_tokens () {
	// if _list_token is null, this means that create_list_token wasn't
	// called, and that this object is still responsible for _tokens.
	// If it isn't null, then that ListToken is responsible for the tokens.
	if (_tokens) {
		for (vector<Token*>::iterator t = _tokens->begin (); t != _tokens->end (); t++) {
			delete *t;
		}
		delete _tokens;
		_tokens = NULL;
	}
}

void Tokens::clear_module_path () {
	if (_module_path) {
		for (vector<const char*>::iterator m = _module_path->begin (); m != _module_path->end (); m++) {
			free (const_cast<char*> (*m));
		}

		delete _module_path;
		_module_path = NULL;
	}
}

void Tokens::clear_attributes () {
	if (_attributes) {
		for (vector<Token*>::iterator a = _attributes->begin (); a != _attributes->end (); a++) {
			delete *a;
		}

		delete _attributes;
		_attributes = NULL;
	}
}

void Tokens::check_attributes () {
	if (_attributes) {
		clear_attributes ();
		ErrorToken *error = new ErrorToken ("unused_attribute");

		error->_sloc = _attribute_sloc;
		_attribute_sloc = NULL;

		push_proper (error);
	}
}

ListToken *Tokens::get_list_token (ListToken::Type type) {
	if (_block_first && _tokens->size () == 1) {
		Token *block = (*_tokens) [0];
		_tokens->clear ();
		assert (dynamic_cast<ListToken*> (block));
		ListToken *list_token = static_cast<ListToken*> (block);
		list_token->_type = type;
		return list_token;
	} else {
		ListToken *list_token = new ListToken (type, get_tokens ());
		list_token->_sloc = _sloc;
		_sloc = NULL;

		return list_token;
	}
}

void Tokens::flush () {
	push_identifier_proper ();
	expecting_attribute ();
	check_attributes ();
}

vector<Token*> *Tokens::get_tokens () {
	flush ();

	vector<Token*> *tokens = _tokens;
	_tokens = NULL;
	return tokens;
}

void Tokens::push_block (Tokens *block, SourceLocation *sloc, bool indent_block) {
	// Blocks are placed on the end of the line, so we defer the newline until
	// after the block has been placed.

	if (!_has_tokens && indent_block)
		_block_first = true;

	push_identifier_proper ();

	check_module_path ();

	ListToken *block_list = block->get_list_token (ListToken::Brace);
	block_list->_sloc = sloc;

	if (_awaiting_attribute) {
		if (!_attributes)
			_attributes = new vector<Token*>;
		_attributes->push_back (block_list);
		_awaiting_attribute = false;
	} else {
		if (_indent_block_last || !indent_block) {
			push_newline_proper ();
		} else {
			_indent_block_last = indent_block && _newline;
		}

		push_proper (block_list);
	}

	if (!indent_block && _newline) {
		_newline = false;
		delete _newline_sloc;
		_newline_sloc = NULL;
	}
}

void Tokens::check_module_path () {
	if (_module_path) {
		clear_module_path ();
		push (new ErrorToken ("missing_module_identifier"), _module_delim_sloc);
		_module_delim_sloc = NULL;
	}
}

void Tokens::push (Token *tok, SourceLocation *sloc) {
	// doesn't matter which order these two are called.
	push_identifier_proper ();
	push_newline_proper ();

	check_module_path ();

	tok->_sloc = sloc;

	if (_awaiting_attribute) {
		if (!_attributes)
			_attributes = new vector<Token*>;
		_attributes->push_back (tok);
		_awaiting_attribute = false;
	} else {
		push_proper (tok);
	}
}

void Tokens::set_attributes (Token *token) {
	if (_attributes) {
		token->_attributes = _attributes;
		_attributes = NULL;
	}
}

void Tokens::set_module (IdentifierToken *id) {
	if (_module_path) {
		id->_module_path = _module_path;
		_module_path = NULL;
	}
}

void Tokens::push_proper (Token *tok) {
	set_attributes (tok);
	assert (tok->_sloc);
	_tokens->push_back (tok);
	_has_tokens = true;
}

void Tokens::push_identifier_proper () {
	if (_last_identifier) {
		IdentifierToken *id = new IdentifierToken (_last_identifier, false);
		_last_identifier = NULL;

		set_module (id);

		id->_sloc = _identifier_sloc;
		_identifier_sloc = NULL;

		if (_awaiting_attribute) {
			if (!_attributes)
				_attributes = new vector<Token*>;
			_attributes->push_back (id);
			_awaiting_attribute = false;
		} else {
			push_proper (id);
		}
	}
}

void Tokens::push_identifier (char *id, SourceLocation *sloc) {
	push_newline_proper ();
	push_identifier_proper ();

	assert (!_identifier_sloc);
	_identifier_sloc = sloc;
	assert (!_last_identifier);
	_last_identifier = id;
}

void Tokens::push_newline_proper () {
	if (_newline) {
		expecting_attribute ();
		check_attributes ();
		check_module_path ();

		IdentifierToken *newline = new IdentifierToken (_strdup (";"), true);

		newline->_sloc = _newline_sloc;
		_newline_sloc = NULL;

		push_proper (newline);
		_newline = false;
		_indent_block_last = false;
	}
}

void Tokens::push_newline (SourceLocation *sloc) {
	push_identifier_proper ();
	expecting_attribute ();
	check_attributes ();

	if (_has_tokens) {
		if (!_newline) {
			assert (!_newline_sloc);
			_newline_sloc = sloc;
		} else {
			// We've already got one, we'll use that, so delete this one.
			delete sloc;
		}
		_newline = true;
	} else {
		delete sloc;
	}
}

void Tokens::push_module_delim (SourceLocation *sloc) {
	if (_last_identifier) {
		if (!_module_path)
			_module_path = new vector<const char*>;
		_module_path->push_back (_last_identifier);
		_last_identifier = NULL;
		delete _identifier_sloc;
		_identifier_sloc = NULL;

		// delete the old one.
		if (_module_delim_sloc)
			delete _module_delim_sloc;

		_module_delim_sloc = sloc;
	} else {
		push (new ErrorToken ("missing_module_identifier"), sloc);
	}
}

void Tokens::expecting_attribute () {
	if (_awaiting_attribute) {
		_awaiting_attribute = false;
		push (new ErrorToken ("no_attribute_found"), _attribute_sloc);
		_attribute_sloc = NULL;
	}
}

void Tokens::await_attribute (SourceLocation *sloc) {
	// doesn't matter which order these two are called.
	push_identifier_proper ();
	push_newline_proper ();

	check_module_path ();

	expecting_attribute ();
	_attribute_sloc = sloc;
	_awaiting_attribute = true;
}

ParserTokens::ParserTokens () {
	_tokens = NULL;
	_sloc = NULL;
}

void ParserTokens::push (Token *tok, SourceLocation *sloc) {
	assert (tok);
	tok->_sloc = sloc;

	if (!_tokens)
		_tokens = new vector<Token*>;

	_tokens->push_back (tok);
}

ListToken *ParserTokens::get_list_token (ListToken::Type type) {
	if (!_tokens)
		_tokens = new vector<Token*>;

	return new ListToken (type, _tokens);
}

NewlineParser::NewlineParser () {
	_newline = false;
	_newline_sloc = NULL;
	_has_tokens = false;
}

void NewlineParser::push (Token *tok, SourceLocation *sloc) {
	if (_newline) {
		ParserTokens::push (new IdentifierToken (_strdup (";"), true), _newline_sloc);

		_newline = false;
		_newline_sloc = NULL;
	}

	ParserTokens::push (tok, sloc);
	_has_tokens = true;
}

void NewlineParser::push_newline (SourceLocation *sloc) {
	if (_has_tokens) {
		_newline = true;
		_newline_sloc = sloc;
	}
}

ModuleParser::ModuleParser () {
	_identifier_sloc = NULL;
	_module_delim_sloc = NULL;
	_last_identifier = NULL;
	_module_path = NULL;
}

void ModuleParser::push_identifier (char *id, SourceLocation *sloc) {
	push_identifier_proper ();

	_last_identifier = id;
	_identifier_sloc = sloc;
}

void ModuleParser::push_module_delim (SourceLocation *sloc) {
	if (_last_identifier) {
		if (!_module_path)
			_module_path = new vector<const char*>;

		_module_path->push_back (_last_identifier);
		_last_identifier = NULL;
		delete _identifier_sloc;
		_identifier_sloc = NULL;

		delete sloc;
	} else {
		NewlineParser::push (new ErrorToken ("module_delimiter_without_identifier"), sloc);
	}
}

void ModuleParser::push (Token *tok, SourceLocation *sloc) {
	push_identifier_proper ();

	NewlineParser::push (tok, sloc);
}

void ModuleParser::push_identifier_proper () {
	if (_last_identifier) {
		IdentifierToken *id = new IdentifierToken (_last_identifier, false);

		if (_module_path) {
			id->_module_path = _module_path;
			_module_path = NULL;
		}

		NewlineParser::push (id, _identifier_sloc);

		_last_identifier = NULL;
		_identifier_sloc = NULL;
	}
}

void ModuleParser::flush () {
	if (_last_identifier) {
		push_identifier_proper ();
	} else if (_module_path) {
		NewlineParser::push (new ErrorToken ("incomplete_module_path"), _module_delim_sloc);
		_module_delim_sloc = NULL;
		delete _module_path;
		_module_path = NULL;
	}
}

AttributeParser::AttributeParser () {
	_awaiting_attribute = false;
	_attribute_sloc = NULL;
	_attributes = NULL;
}

void AttributeParser::check_awaiting_attribute () {
	if (_awaiting_attribute) {
		_awaiting_attribute = false;
		ModuleParser::push (new ErrorToken ("no_attribute"), _attribute_sloc);
		_attribute_sloc = NULL;
	}
}

void AttributeParser::await_attribute (SourceLocation *sloc) {
	check_awaiting_attribute ();

	_awaiting_attribute = true;
	_attribute_sloc = sloc;
}

void AttributeParser::push (Token *tok, SourceLocation *sloc) {
	if (_awaiting_attribute) {
		if (!_attributes)
			_attributes = new vector<Token*>;

		tok->_sloc = sloc;
		_attributes->push_back (tok);

		_awaiting_attribute = false;
		delete _attribute_sloc;
		_attribute_sloc = NULL;
	} else {
		ModuleParser::push (tok, sloc);
	}
}

void AttributeParser::flush () {
	check_awaiting_attribute ();

	ModuleParser::flush ();
}

BlockParser::BlockParser () {
	_solitair_block_sloc = NULL;
	_solitair_block = NULL;
	_has_tokens = false;
}

void BlockParser::push_block (Parser *block, SourceLocation *sloc) {
	if (!_has_tokens) {
		_solitair_block = block;
		_solitair_block_sloc = sloc;
	} else {
		AttributeParser::push (block->get_list_token (ListToken::Brace), sloc);
	}

	_has_tokens = true;
}

void BlockParser::push (Token *tok, SourceLocation *sloc) {
	if (_solitair_block) {
		AttributeParser::push (_solitair_block->get_list_token (ListToken::Brace), _solitair_block_sloc);
	}

	AttributeParser::push (tok, sloc);

	_has_tokens = true;
}

void BlockParser::flush () {
	if (_solitair_block) {
		_solitair_block->flush ();
	}

	AttributeParser::flush ();
}

ListToken *BlockParser::get_list_token (ListToken::Type type) {
	flush ();

	if (_solitair_block) {
		return _solitair_block->get_list_token (type);
	} else {
		return AttributeParser::get_list_token (type);
	}
}

Lexer::Lexer (Block *block) {
	_lineno = block->_lineno;
	_colno = block->_colno;
	_lines = block->_first;
	_source = block->_source;
	_file_name = block->_file_name;
}

void Lexer::consume_dots (Parser *parser, TokenLocation *token_loc, char **line) {
	int index = 0;
	int c;

	token_buffer [index++] = **line;

	c = *++*line;
	while (c == '.' && index < TOKEN_BUFFER_LENGTH) {
		token_buffer [index++] = c;
		c = *++*line;
	}

	token_buffer [index] = 0;

	parser->push (new IdentifierToken (_strdup (token_buffer), true), token_loc->calc_location (*line));
}

void Lexer::consume_operator (Parser *parser, TokenLocation *token_loc, char **line) {
	int index = 0;
	int c;

	token_buffer [index++] = **line;

	c = *++*line;
	while (is_operator (c) && index < TOKEN_BUFFER_LENGTH) {
		token_buffer [index++] = c;
		c = *++*line;
	}

	token_buffer [index] = 0;

	parser->push (new IdentifierToken (_strdup (token_buffer), true), token_loc->calc_location (*line));
}

void consume_identifier_token (char **line, char *buffer, uint length) {
	uint index = 0;
	int c;

	buffer [index++] = **line;

	c = *++*line;
	while (is_identifier (c) && index < length) {
		buffer [index++] = c;
		c = *++*line;
	}

	buffer [index] = 0;
}

void Lexer::consume_identifier (Parser *parser, TokenLocation *token_loc, char **line) {
	consume_identifier_token (line, token_buffer, TOKEN_BUFFER_LENGTH);

	parser->push_identifier (_strdup (token_buffer), token_loc->calc_location (*line));
	//parser->push (new IdentifierToken (_strdup (token_buffer)), token_loc->calc_location (*line));
	//parser->push_proper (new IdentifierToken (_strdup (token_buffer)));
}

bool is_quote_start (int c) {
	return is_operator (c)
		|| c == '"'
		|| c == '\''
		|| c == '{'
		|| c == '('
		|| c == '[';
}

int get_quote_end (int c) {
	if (is_operator (c) || c == '\'' || c == '"')
		return c;
	else if (c == '{')
		return '}';
	else if (c == '(')
		return ')';
	else if (c == '[')
		return '[';
	else {
		assert (false);
		return c;
	}
}

void Lexer::consume_quote (Parser *parser, TokenLocation *token_loc, char **line) {
	int index = 0;

	int c = *++*line;
	char *label;
	while (is_identifier (c) && index < TOKEN_BUFFER_LENGTH) {
		token_buffer [index++] = c;
		c = *++*line;
	}
	token_buffer [index] = 0;

	label = _strdup (token_buffer);

	if (is_quote_start (c)) {
		int end = get_quote_end (c);
		consume_quote_string (parser, token_loc, line, label, end, true);
	} else {
		parser->push (new ErrorToken ("invalid_quote_character"), token_loc->calc_location (*line));
	}
}

void Lexer::consume_quote_string (Parser *parser, TokenLocation *token_loc, char **line, char *label, char end, bool escaped) {
	int index = 0;

	while (index < TOKEN_BUFFER_LENGTH) {
		int c = *++*line;

		if (c == 0) {
			break;
		} else if (!escaped && c == end && (*line)[1] == end) {
			token_buffer [index++] = end;
			++*line;
		} else if (c == end) {
			++*line;
			break;
		} else if (escaped && c == '\\' && (*line)[1] == end) {
			token_buffer [index++] = end;
			++*line;
		} else if (escaped && c == '\\' && (*line)[1] == '\\') {
			token_buffer [index++] = '\\';
			token_buffer [index++] = '\\';
			++*line;
		} else
			token_buffer [index++] = c;
	}
	token_buffer [index] = 0;

	parser->push (new StringToken (label, _strdup (token_buffer), escaped), token_loc->calc_location (*line));
}

void Lexer::consume_multiline_quote (Parser *parser, char **line, TokenLocation *token_loc) {
	*line += 2;
	Block contents;

	do {
		const char *line_start = *line;

		while (**line) {
			if (**line == ':' && (*line)[1] == '}') {
				char *new_line = make_line (line_start, *line);
				contents.add_line (0, new_line, new_line, false);

				*line += 2;
				parser->push (new StringToken (_strdup ("quote"), contents.get_content (false), false),
							  token_loc->calc_location (*line, _lineno, _line_start));
				return;
			}

			++*line;
		}

		char *new_line = _strdup (line_start);
		contents.add_line (0, new_line, new_line, false);

		next_line ();

		if (_lines)
			*line = _lines->_whole_line;
		else
			break;
	} while (true);

	// comment not closed!
	parser->push (new ErrorToken ("unclosed_quote"), token_loc->calc_location (*line, _lineno, _line_start));
}

bool Lexer::consume_suffix (Parser *parser, TokenLocation *token_loc, char **line, long integer) {
	if (is_word (**line)) {
		int index = 0;

		do {
			token_buffer [index++] = **line;
			(*line)++;
		} while (is_word (**line) && index < TOKEN_BUFFER_LENGTH);
		token_buffer [index] = 0;

		parser->push (new ErrorToken ("bad_integer"), token_loc->calc_location (*line));

		return true;
	} else
		return false;
}

void Lexer::consume_hex_number (Parser *parser, TokenLocation *token_loc, char **line) {
	long n = 0;
	char c;

	++*line;
	while (is_hex (c = *++*line)) {
		int value;

		if (c >= 'a' && c <= 'f')
			value = 10 + c - 'a';
		else if (c >= 'A' && c <= 'F')
			value = 10 + c - 'A';
		else
			value = c - '0';

		n = n * 16 + value;
	}

	if (!consume_suffix (parser, token_loc, line, n))
		parser->push (new IntegerToken (n), token_loc->calc_location (*line));
}

void Lexer::consume_binary_number (Parser *parser, TokenLocation *token_loc, char **line) {
	long n = 0;
	char c;

	++*line;

	while (is_binary (c = *++*line))
		n = n * 2 + c - '0';

	if (!consume_suffix (parser, token_loc, line, n))
		parser->push (new IntegerToken (n), token_loc->calc_location (*line));
}

void Lexer::consume_octal_number (Parser *parser, TokenLocation *token_loc, char **line) {
	long n = 0;
	char c;

	++*line;
	while (is_octal (c = *++*line))
		n = n * 8 + c - '0';

	if (!consume_suffix (parser, token_loc, line, n))
		parser->push (new IntegerToken (n), token_loc->calc_location (*line));
}

void Lexer::consume_decimal_number (Parser *parser, TokenLocation *token_loc, char **line) {
	long n = 0;
	char c;

	while (is_numeric (c = **line)) {
		n = n * 10 + c - '0';
		++*line;
	}

	if (c == 'e' || c == 'E' || c == '.')
		consume_floating_number (parser, token_loc, line, n);
	else if (!consume_suffix (parser, token_loc, line, n))
		parser->push (new IntegerToken (n), token_loc->calc_location (*line));
}

void Lexer::consume_floating_number (Parser *parser, TokenLocation *token_loc, char **line, long integral_part) {
	double fractional = 0.0;
	double value;
	bool has_fractional = false;

	if (**line == '.') {
		char c;
		double place = 1.0;
		while (is_numeric (c = *++*line)) {
			place /= 10;
			fractional = fractional + place * (c - '0');
			has_fractional = true;
		}

		if (!has_fractional) {
			(*line)--;

			parser->push (new IntegerToken (integral_part), token_loc->calc_location (*line));

			return;
		}
	}

	value = ((double) integral_part) + fractional;

	if (**line == 'e' || **line == 'E') {
		long exponent = 0;

		char c;
		while (is_numeric (c = *++*line)) {
			exponent = exponent * 10 + c - '0';
		}

		value *= exp (log (10.0) * exponent);
	}

	if (!consume_suffix (parser, token_loc, line, 0))
		parser->push (new FloatToken (value), token_loc->calc_location (*line));
}

long consume_decimal_segment (char **line) {
	long n = 0;
	char c;

	while (is_numeric (c = **line)) {
		n = n * 10 + c - '0';
		++*line;
	}

	return n;
}

void consume_version_segment (char **line, Version *version) {
	version->major = consume_decimal_segment (line);
	if (**line == '.') {
		++*line;
		version->minor = consume_decimal_segment (line);
	}
	if (**line == '.') {
		++*line;
		version->revision = consume_decimal_segment (line);
	}
	if (**line == '.') {
		++*line;
		version->build = consume_decimal_segment (line);
	}
}

void Lexer::consume_version (Parser *parser, TokenLocation *token_loc, char **line) {
	VersionToken *version = new VersionToken ();

	++*line;

	consume_version_segment (line, &version->_version);

	if (**line == '-') {
		++*line;
		version->_has_to_version = true;
		consume_version_segment (line, &version->_to_version);
	}

	if (**line == '@') {
		++*line;
		consume_identifier_token (line, token_buffer, TOKEN_BUFFER_LENGTH);
		version->_branch = _strdup (token_buffer);
	}

	parser->push (version, token_loc->calc_location (*line));
}

void Lexer::consume_number (Parser *parser, TokenLocation *token_loc, char **line) {
	int c = **line;
	if (c == '0') {
		switch ((*line)[1]) {
			case 'b':
			case 'B':
				++*line;
				consume_binary_number (parser, token_loc, line);
				return;
			case 'x':
			case 'X':
				++*line;
				consume_hex_number (parser, token_loc, line);
				return;
			case 'v':
			case 'V':
				++*line;
				consume_version (parser, token_loc, line);
				return;
			case 'o':
			case 'O':
				++*line;
				consume_octal_number (parser, token_loc, line);
				return;
		}
	}

	consume_decimal_number (parser, token_loc, line);
}

void Lexer::next_line () {
	_lines = _lines->_next;
	if (_lines) {
		_lineno++;
		_line_start = _lines->_whole_line;
	}
}

char *Lexer::get_quote_block () {
	Block sub_block;

	while (_lines) {
		char *line = _lines->_whole_line;
		while (is_whitespace (*line)) {
			line++;
		}

		if (*line == ':') {
			line++;
			char *sub_line = _strdup (line);
			sub_block.add_line (0, sub_line, sub_line, false);
		} else {
			break;
		}

		next_line ();
	}

	return sub_block.get_content (true);
}

char *Lexer::get_indented_block (uint indent) {
	Block sub_block;

	sub_block._lineno = _lineno;
	sub_block._colno = _colno + 1;
	sub_block._source = _source;

	uint min_indent = static_cast<uint> (-1);
	uint n_empty_lines = 0;

	// while we've got indented lines, or lines with 0 length; add them
	// to the block.
	while ((_lines && _lines->_indent > indent || (_lines && !(*_lines->_line) && _lines->_next)) && !_lines->_indent_error) {
		if (*_lines->_line) {
			while (n_empty_lines) {
				n_empty_lines--;
				char *empty = _strdup ("");
				sub_block.add_line (0, empty, empty, false);
			}

			if (min_indent > _lines->_indent) {
				min_indent = _lines->_indent;
			}

			sub_block.add_line (_lines->clone ());
		} else {
			n_empty_lines++;
		}
		next_line ();
	}

	sub_block.set_indent (min_indent);
	return sub_block.get_content (true);
}

char get_matching_bracket (char c) {
	switch (c) {
	case '{':
		return '}';
	case '(':
		return ')';
	case '[':
		return ']';
	default:
		assert (false);
		return 0;
	}
}

ListToken::Type get_bracket_list_type (char c) {
	switch (c) {
	case '{':
		return ListToken::Brace;
	case '(':
		return ListToken::Parenthesis;
	case '[':
		return ListToken::Bracket;
	default:
		assert (false);
		return ListToken::List;
	}
}

void Lexer::consume_multiline_comment (Parser *parser, char **line, TokenLocation *token_loc) {
	*line += 2;

	do {
		while (**line) {
			if (**line == '*' && (*line)[1] == '/') {
				*line += 2;
				return;
			}

			++*line;
		}
		next_line ();

		if (_lines)
			*line = _lines->_line;
		else
			break;
	} while (true);

	// comment not closed!
	parser->push (new ErrorToken ("unclosed_comment"), token_loc->calc_location (*line, _lineno, _line_start));
}

void Lexer::reset_counters (Parser *parser, TokenLocation *token_loc, char bracket_end) {
	if (_lines) {
		token_loc->_lineno = _lineno;
		token_loc->_line_start = _lines->_whole_line;

		if (_lines->_indent_error && !bracket_end) {
			token_loc->_token_start = _lines->_whole_line;
			if (parser)
				parser->push (new ErrorToken ("indentation_error"), token_loc->calc_location (_lines->_line));
		}
	}
}

void Lexer::lex (Parser *parser, char **line, char bracket_end, bool in_block, uint min_indent, uint indent, bool single_token) {
	TokenLocation token_loc;
	token_loc._colno = _colno;
	token_loc._source = _source;
	token_loc._file_name = _file_name;
	token_loc._token_start = *line;
	_line_start = *line;

	while (_lines) {
		int c;

		if (_lines->_indent > indent && *_lines->_line && !single_token && !bracket_end) {
			Parser block_parser;
			*line = _lines->_line;

			token_loc._lineno = _lineno;
			token_loc._line_start = _lines->_whole_line;
			token_loc._token_start = *line;

			lex (&block_parser, line, bracket_end, true, min_indent, _lines->_indent, false);

			parser->push_block (&block_parser, token_loc.calc_location (*line, _lineno, _line_start), true);

			if (!_lines && !**line)
				break;
			else if (!**line)
				*line = _lines->_line;
		}
		
		if (_lines->_indent < indent && *_lines->_line && !single_token && _lines->_indent >= min_indent && !bracket_end)
			break;

		reset_counters (parser, &token_loc, bracket_end);

		while ((c = **line)) {
			token_loc._token_start = *line;

			if (c == '.') {
				consume_dots (parser, &token_loc, line);
			} else if (c == ',') {
				++*line;
				parser->push (new IdentifierToken (_strdup (","), true), token_loc.calc_location (*line));
			} else if (c == ';') {
				++*line;
				parser->push (new IdentifierToken (_strdup (";"), true), token_loc.calc_location (*line));
			} else if (c == '@') {
				// attributes
				++*line;
				parser->await_attribute (token_loc.calc_location (*line));
			} else if (c == ':') {
				if (is_operator (*(*line + 1))) {
					consume_operator (parser, &token_loc, line);
				} else {
					++*line;

					while (is_whitespace (**line)) {++*line;}
					if (!**line && !bracket_end) {
						int newline_lineno = _lineno;
						int newline_colno = *line - token_loc._line_start;

						next_line ();

						char *content;
						content = get_indented_block (indent);

						parser->push (new StringToken (_strdup ("double_quote"), content, false), token_loc.calc_location (*line));
						parser->push_newline (TokenLocation::new_location (_source, _file_name, newline_lineno, newline_colno));
						if (!_lines)
							break;
						else
							*line = _lines->_line;

						reset_counters (parser, &token_loc, bracket_end);
					} else {
						parser->push_module_delim (token_loc.calc_location (*line));
					}
				}
			} else if (c == '%') {
				consume_quote (parser, &token_loc, line);
			} else if (c == '"') {
				consume_quote_string (parser, &token_loc, line, _strdup ("double_quote"), '"', true);
			} else if (c == '{' || c == '(' || c == '[') {
				if ((*line)[1] == ':') {
					consume_multiline_quote (parser, line, &token_loc);
					reset_counters (parser, &token_loc, bracket_end);
				} else {
					Parser subparser;
					++*line;
					lex (&subparser, line, get_matching_bracket (c), false, indent, indent, false);
					if (c == '{') {
						parser->push_block (&subparser, token_loc.calc_location (*line, _lineno, _line_start), false);
					} else {
						parser->push (subparser.get_list_token (get_bracket_list_type (c)), token_loc.calc_location (*line, _lineno, _line_start));
					}
				}
			} else if (c == '\'') {
				consume_quote_string (parser, &token_loc, line, _strdup ("single_quote"), '\'', false);
			} else if (is_whitespace (c)) {
				if (!bracket_end) {
					if (_lines->_line == *line)
						parser->push (new ErrorToken ("space_at_start_of_line"), token_loc.calc_location (*line));
				}

				while (is_whitespace (*++*line));
			} else if (is_numeric (c)) {
				consume_number (parser, &token_loc, line);
			} else if (is_first_identifier (c)) {
				consume_identifier (parser, &token_loc, line);
			} else if (c == '/' && (*line)[1] == '*') {
				consume_multiline_comment (parser, line, &token_loc);
				reset_counters (parser, &token_loc, bracket_end);
			} else if (c == '/' && (*line)[1] == '/') {
				parser->push_newline (token_loc.calc_location (*line));
				next_line ();

				if (!_lines)
					break;
				else
					*line = _lines->_line;

				reset_counters (parser, &token_loc, bracket_end);
			} else if (is_operator (c)) {
				consume_operator (parser, &token_loc, line);
			} else if (c == bracket_end) {
				// we've seen the end of a bracketted section -- { a b c }
				// now just return
				if (!in_block)
					++*line;
				return;
			} else if (c == '}' || c == ')' || c == ']') {
				++*line;
				parser->push (new ErrorToken ("unmatched_closing_bracket"), token_loc.calc_location (*line));
			} else {
				++*line;
				parser->push (new ErrorToken ("invalid_character"), token_loc.calc_location (*line));
			}

			if (single_token)
				return;
		}

		if (_lines) {
			int end_of_line_no = _lineno;
			int end_of_line_col = *line - token_loc._line_start;

			next_line ();

			if (_lines) {
				*line = _lines->_line;
				parser->push_newline (TokenLocation::new_location (_source, _file_name, end_of_line_no, end_of_line_col));
			}
		}
	}

	if (bracket_end && !in_block)
		parser->push (new ErrorToken ("unmatched_opening_bracket"), token_loc.calc_location (*line));
}

void Lexer::interpolate (vector<Token*> *items) {
	string str;
	StringToken *string_token = NULL;

	TokenLocation token_loc;
	token_loc._colno = _colno;
	token_loc._source = _source;
	token_loc._file_name = _file_name;

	assert (_lines);
	char *line = _lines->_whole_line;

	while (_lines) {
		line = _lines->_whole_line;
		char *str_begin = line;

		reset_counters (NULL, &token_loc, 0);

		while (*line) {
			if (*line == '\\' && line[1] == INTERPOLATION_ESCAPE) {
				line += 2;
			} else if (*line == INTERPOLATION_ESCAPE) {
				if (str.size () > 0) {
					str.append (str_begin, line - str_begin);
					string_token = new StringToken (_strdup ("quote"), _strdup (str.c_str ()), true);
					string_token->_sloc = token_loc.calc_location (line, _lineno, _line_start);
					items->push_back (string_token);
					str.clear ();
				} else if (line - str_begin > 0) {
					string_token = new StringToken (_strdup ("quote"), strndup (str_begin, line - str_begin), true);
					string_token->_sloc = token_loc.calc_location (line, _lineno, _line_start);
					items->push_back (string_token);
				}

				Parser parser;

				line++;
				lex (&parser, &line, 0, false, _lines->_indent, _lines->_indent, true);

				vector<Token*> *token_vector = parser.get_tokens ();

				long n_tokens = token_vector->size ();
				assert (n_tokens <= 1);

				if (n_tokens == 1) {
					items->push_back (token_vector->at (0));
					token_vector->pop_back ();
				} else
					items->push_back (new ErrorToken ("no_interpolated_token"));

				delete token_vector;

				str_begin = line;
			} else if (*line) {
				line++;
			}
		}

		if (line - str_begin > 0) {
			str.append (str_begin, line - str_begin);
		}

		if (_lines) {
			next_line ();

			if (_lines)
				str += "\n";
		}
	}

	if (str.size () > 0) {
		string_token = new StringToken (_strdup ("quote"), _strdup (str.c_str ()), true);
		string_token->_sloc = token_loc.calc_location (line, _lineno, _line_start);
		items->push_back (string_token);
	}
}

#ifdef JAVA
NoClassException::NoClassException (const char *clsName) {
	_clsName = clsName;
}

jclass JavaEnvironment::getClass (JNIEnv *env, const char *clsName) {
	jclass cls = env->FindClass (clsName);

	if (!cls) {
		throw NoClassException (clsName);
	}

	return cls;
}

jmethodID JavaEnvironment::getMethod (JNIEnv *env, jclass cls, const char *name, const char *sig) {
	jmethodID mid = env->GetMethodID (cls, name, sig);

	if (!mid) {
		throw NoClassException (sig);
	}

	return mid;
}

jmethodID JavaEnvironment::getConstructor (JNIEnv *env, jclass cls, const char *sig) {
	return getMethod (env, cls, "<init>", sig);
}

JavaEnvironment::JavaEnvironment (JNIEnv *env) {
	_jni = env;

	_sourceDescriptionClass = getClass (_jni, "org/gnostix/indra/parser/SourceDescription");
	_sourceLocationClass = getClass (_jni, "org/gnostix/indra/parser/SourceLocation");
	_integerTokenClass = getClass (_jni, "org/gnostix/indra/parser/IntegerToken");
	_identifierTokenClass = getClass (_jni, "org/gnostix/indra/parser/IdentifierToken");
	_bracketTokenClass = getClass (_jni, "org/gnostix/indra/parser/BracketToken");
	_arrayListClass = getClass (_jni, "java/util/ArrayList");
	_floatingTokenClass = getClass (_jni, "org/gnostix/indra/parser/FloatingToken");
	_stringTokenClass = getClass (_jni, "org/gnostix/indra/parser/StringToken");
	_quoteTokenClass = getClass (_jni, "org/gnostix/indra/parser/QuoteToken");
	_errorTokenClass = getClass (_jni, "org/gnostix/indra/parser/ErrorToken");
	_versionTokenClass = getClass (_jni, "org/gnostix/indra/parser/VersionToken");
	_versionClass = getClass (_jni, "org/gnostix/indra/parser/Version");
	_nullPointerExceptionClass = getClass (_jni, "java/lang/NullPointerException");

	_sourceDescriptionConstructor = getConstructor (_jni, _sourceDescriptionClass, "(ZLjava/lang/String;)V");
	_sourceDescriptionGetIsFilename = getMethod (_jni, _sourceDescriptionClass, "getIsFilename", "()Z");
	_sourceDescriptionGetFilename = getMethod (_jni, _sourceDescriptionClass, "getFilename", "()Ljava/lang/String;");
	_sourceLocationConstructor = getConstructor (_jni, _sourceLocationClass, "(Lorg/gnostix/indra/parser/SourceDescription;IIII)V");
	_integerTokenConstructor = getConstructor (_jni, _integerTokenClass, "(JLjava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_identifierTokenConstructor = getConstructor (_jni, _identifierTokenClass, "(Ljava/lang/String;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_bracketTokenConstructor = getConstructor (_jni, _bracketTokenClass, "(ILjava/util/List;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_arrayListConstructor = getConstructor (_jni, _arrayListClass, "()V");
	_arrayListAdd = getMethod (_jni, _arrayListClass, "add", "(Ljava/lang/Object;)Z");
	_floatingTokenConstructor = getConstructor (_jni, _floatingTokenClass, "(DLjava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_stringTokenConstructor = getConstructor (_jni, _stringTokenClass, "(Ljava/lang/String;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_quoteTokenConstructor = getConstructor (_jni, _quoteTokenClass, "(Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_errorTokenConstructor = getConstructor (_jni, _errorTokenClass, "(Ljava/lang/String;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_versionTokenConstructor1 = getConstructor (_jni, _versionTokenClass, "(Lorg/gnostix/indra/parser/Version;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_versionTokenConstructor2 = getConstructor (_jni, _versionTokenClass, "(Lorg/gnostix/indra/parser/Version;Lorg/gnostix/indra/parser/Version;Ljava/lang/String;Ljava/util/List;Lorg/gnostix/indra/parser/SourceLocation;)V");
	_versionConstructor = getConstructor (_jni, _versionClass, "(IIII)V");
}

jstring JavaEnvironment::createString (const char *str) {
	return _jni->NewStringUTF (str);
}

const char *JavaEnvironment::getChars (jstring str) {
	return _jni->GetStringUTFChars (str, NULL);
}

void JavaEnvironment::releaseChars (jstring str, const char *chars) {
	_jni->ReleaseStringUTFChars (str, chars);
}

jobject JavaEnvironment::getSourceDescription () {
	return _sourceDescription;
}

void JavaEnvironment::setSourceDescription (jobject desc) {
	_sourceDescription = desc;
}

char *JavaEnvironment::setBlockSource (jstring str) {
	_source = str;
	return _strdup (getChars (str));
}

jstring JavaEnvironment::getBlockSource () {
	return _source;
}

jobject JavaEnvironment::createSourceDescription (jboolean isFilename, jstring filename) {
	return _jni->NewObject (_sourceDescriptionClass, _sourceDescriptionConstructor, isFilename, filename);
}

jstring JavaEnvironment::SourceDescription_GetFilename (jobject srcDesc) {
	return (jstring) _jni->CallObjectMethod (srcDesc, _sourceDescriptionGetFilename);
}

jboolean JavaEnvironment::SourceDescription_GetIsFilename (jobject srcDesc) {
	return _jni->CallBooleanMethod (srcDesc, _sourceDescriptionGetIsFilename);
}

jobject JavaEnvironment::createSourceLocation (jobject desc, jint rowStart, jint rowEnd, jint colStart, jint colEnd) {
	return _jni->NewObject (_sourceLocationClass, _sourceLocationConstructor, desc, rowStart, rowEnd, colStart, colEnd);
}

jobject JavaEnvironment::createIntegerToken (jlong value, jobject attrs, jobject sloc) {
	return _jni->NewObject (_integerTokenClass, _integerTokenConstructor, value, attrs, sloc);
}

jobject JavaEnvironment::createIdentifierToken (jstring id, jobject attrs, jobject sloc) {
	return _jni->NewObject (_identifierTokenClass, _identifierTokenConstructor, id, attrs, sloc);
}

jobject JavaEnvironment::createBracketToken (jint type, jobject tokens, jobject attrs, jobject sloc) {
	return _jni->NewObject (_bracketTokenClass, _bracketTokenConstructor, type, tokens, attrs, sloc);
}

jobject JavaEnvironment::createArrayList () {
	return _jni->NewObject (_arrayListClass, _arrayListConstructor);
}

void JavaEnvironment::ArrayList_Add (jobject ary, jobject item) {
	_jni->CallBooleanMethod (ary, _arrayListAdd, item);
}

jobject JavaEnvironment::createFloatingToken (jdouble value, jobject attrs, jobject sloc) {
	return _jni->NewObject (_floatingTokenClass, _floatingTokenConstructor, value, attrs, sloc);
}

jobject JavaEnvironment::createStringToken (jstring value, jobject attrs, jobject sloc) {
	return _jni->NewObject (_stringTokenClass, _stringTokenConstructor, value, attrs, sloc);
}

jobject JavaEnvironment::createQuoteToken (jstring type, jstring value, jobject attrs, jobject sloc) {
	return _jni->NewObject (_quoteTokenClass, _quoteTokenConstructor, type, value, attrs, sloc);
}

jobject JavaEnvironment::createErrorToken (jstring error, jobject attrs, jobject sloc) {
	return _jni->NewObject (_errorTokenClass, _errorTokenConstructor, error, attrs, sloc);
}

jobject JavaEnvironment::createVersionToken (jobject version, jobject attrs, jobject sloc) {
	return _jni->NewObject (_versionTokenClass, _versionTokenConstructor1, version, attrs, sloc);
}

jobject JavaEnvironment::createVersionToken (jobject version, jobject toVersion, jstring branch, jobject attrs, jobject sloc) {
	return _jni->NewObject (_versionTokenClass, _versionTokenConstructor2, version, toVersion, branch, attrs, sloc);
}

jobject JavaEnvironment::createVersion (jint major, jint minor, jint revision, jint build) {
	return _jni->NewObject (_versionClass, _versionConstructor, major, minor, revision, build);
}

void JavaEnvironment::throwNullPointerException (const char *msg) {
	_jni->ThrowNew (_nullPointerExceptionClass, msg);
}
#endif